# -*- coding: utf-8 -*-




import scrapy
from scrapy import Spider
from scrapy import Request, FormRequest
import datetime
from scrapyluke.uuCode import UUcode
from scrapyluke.commonfun import to_str
from scrapyluke.processors import html_to_dict, html_to_list_dict, tables_names
from scrapyluke.items import MongoItem
from gsxt_gov.spiders.report_format import tables_to_dict
from gsxt_gov.items import GsxtGovItem
import re
import hashlib
import time
import urlparse

# 已经加入分词，但是未测试




class jiangxiSpider(scrapy.Spider):

    name = 'jiangxi_test'
    #start_urls = ['http://gsxt.jxaic.gov.cn/ECPS/']



    def __init__(self):
        super(jiangxiSpider, self).__init__()
        self.uu = UUcode('109521', '3c8906d3666e44bb9c961e1647126dbc')
        self.code_type = 8001
        self.code_getMax = 30
        self.host = 'http://gsxt.jxaic.gov.cn'
        # 该省份分词读取路径


        self.ppl_path = '/home/li/working/fenci'
        # 实时记录跑过的词语


        self.realtime_record_path = '/home/li/working/finished_fenci_jiangxi'
        # 记录本次运行跳过的词语(上次已经跑过的词语)


        self.skip_record_path = '/home/li/working/finished_already_jiangxi'

        self.already = self.wordcount()
        self.words = self.creatlist(1, 10)

    def start_requests(self):
        for i in self.words:
            yield Request(url=self.host + '/ECPS/', meta={'cookiejar': i, 'keyword': i}, dont_filter='true')

    def code_right(self, res):
        if '\"msg\":\"true\"' in res:
            return True
        return

    def parse(self, response):
        yield Request(url=self.host + '/ECPS/common/common_getJjYzmImg.pt?yzmName=searchYzm&imgWidth=180',
                      meta={'cookiejar': response.meta['keyword'], 'keyword': response.meta['keyword']},
                      callback=self.code_ocr,
                      dont_filter='true')

    def code_ocr(self, response):
        result = self.uu.check_code_cache(response.body, self.code_type, self.code_getMax)
        if result:
            id, code = result
            # 判断验证码请求


            yield scrapy.FormRequest(url=self.host + '/ECPS/home/home_homeSearchYzm.pt',
                                     method='POST',
                                     formdata={'yzm': code,'searchtext':response.meta['keyword']},
                                     meta={'cookiejar':response.meta['keyword'], 'code': code, 'code_id': id, 'keyword': response.meta['keyword']},
                                     callback=self.check_code,
                                     dont_filter='true')

    def check_code(self, response):
        if self.code_right(response.body):
            # get_url = self.host + '/ECPS/home/home_homeSearch.pt?yzm=' + response.meta['code'] + '&search=' + response.meta['keyword']


            # yield scrapy.Request(url =get_url,callback=self.get_infolist)


            data = {
                'searchtext':response.meta['keyword'],
                'pageFlag':'1',
                'limit': '50',
                'page':'1',
                'yzm':response.meta['code']
            }
            yield scrapy.FormRequest(url=self.host + '/ECPS/home/home_homeSearch.pt?searchPage=yes',
                                     formdata=data,
                                     meta={'cookiejar':response.meta['keyword'], 'keyword':response.meta['keyword']},
                                     callback=self.get_infolist,
                                     dont_filter='true')
        else:
            # 提交验证码错误信息(费用会被退还)


            self.uu.report_error(response.meta['code_id'])

    def get_infolist(self, response):
        url_list = response.xpath("//dl/div/a/@href").extract()
        name_list = response.xpath("//dl/div/a//text()").extract()
        zip(url_list, name_list)
        list(zip(url_list, name_list))

        listname = ''
        # 将搜索列表记录


        op = open(self.realtime_record_path,'a')
        op.write(response.request.meta['keyword']+':'+ listname.join([i+'-' for i in name_list]) +','+str(datetime.datetime.now())+','+response.url+'\n')
        op.close()

        for (i, name) in zip(url_list, name_list):
            query = urlparse.urlparse(i).query
            ps = dict([(k,v[0]) for k,v in urlparse.parse_qs(query).items()])
            data = {
                'qymc':name,
                'qyid':ps['qyid'],
                'qylx':ps['qylx'],
                'zch':ps['zch'],
                'cookiejar':response.meta['keyword'],
                'keyword':response.meta['keyword']
            }
            yield scrapy.Request(url = i, meta=data, callback=self.get_allinfo, dont_filter='true')

    def get_allinfo(self, response):
        qymc = response.meta['qymc']
        qyid = response.meta['qyid']
        qylx = response.meta['qylx']
        zch = response.meta['zch']
        num = '0'
        showgdxx = 'true'
        # 工商公示


        # 基本信息


        yield Request(url=self.host+'/ECPS/ccjcgs/gsgs_viewDjxx.pt?qyid='+qyid+'&zch='+zch+"&qylx="+qylx+"&num="+num+"&showgdxx=true",
                      meta={'qymc':qymc, 'cookiejar':response.meta['keyword'], 'keyword':response.meta['keyword']},
                      callback=self.get_table_1,
                      dont_filter='true')
        # 发起人信息


        yield Request(url=self.host+'/ECPS/ccjcgs/gsgs_viewDjxxGdxx.pt?qyid='+qyid+'&zch='+zch+'&qylx='+qylx,
                      meta={'qymc':qymc, 'cookiejar':response.meta['keyword'], 'keyword':response.meta['keyword']},
                      callback=self.get_table_2,
                      dont_filter='true')
        # 变更信息


        yield Request(url=self.host+'/ECPS/ccjcgs/gsgs_viewDjxxBgxx.pt?qyid='+qyid,
                      meta={'qymc':qymc, 'cookiejar':response.meta['keyword'], 'keyword':response.meta['keyword']},
                      callback=self.get_table_3,
                      dont_filter='true')
        # 备案信息


        yield Request(url=self.host+"/ECPS/ccjcgs/gsgs_viewBaxx.pt?qyid="+qyid+"&zch="+zch+"&qylx="+qylx+"&showgdxx=true",
                      meta={'qymc':qymc, 'cookiejar':response.meta['keyword'], 'keyword':response.meta['keyword']},
                      callback=self.get_table_4,
                      dont_filter='true')
        # 股权出质登记


        yield Request(url=self.host+"/ECPS/ccjcgs/gsgs_viewGqczdjxx.pt?qyid="+qyid+"&zch="+zch+"&qylx="+qylx+"&showgdxx=true",
                      meta={'qymc':qymc, 'cookiejar':response.meta['keyword'], 'keyword':response.meta['keyword']},
                      callback=self.get_table_5,
                      dont_filter='true')
        # 动产抵押


        yield Request(url=self.host+"/ECPS/ccjcgs/gsgs_viewDcdydjxx.pt?qyid="+qyid+"&zch="+zch+"&qylx="+qylx+"&num="+num+"&showgdxx=true",
                      meta={'qymc':qymc, 'cookiejar':response.meta['keyword'], 'keyword':response.meta['keyword']},
                      callback=self.get_table_6,
                      dont_filter='true')
        # 经营异常


        yield Request(url=self.host+"/ECPS/ccjcgs/gsgs_viewJyycxx.pt?qyid="+qyid+"&zch="+zch+"&qylx="+qylx+"&showgdxx=true",
                      meta={'qymc':qymc, 'cookiejar':response.meta['keyword'], 'keyword':response.meta['keyword']},
                      callback=self.get_table_7,
                      dont_filter='true')
        # 严重违法信息



        yield Request(url=self.host+"/ECPS/ccjcgs/gsgs_viewYzwfxx.pt?qyid="+qyid+"&zch="+zch+"&qylx="+qylx+"&showgdxx=true",
                      meta={'qymc':qymc, 'cookiejar':response.meta['keyword'], 'keyword':response.meta['keyword']},
                      callback=self.get_table_8,
                      dont_filter='true')
        # 行政处罚信息


        yield Request(url=self.host+"/ECPS/ccjcgs/gsgs_viewXzcfxx.pt?qyid="+qyid+"&zch="+zch+"&qylx="+qylx+"&num="+num+"&showgdxx=true",
                      meta={'qymc':qymc, 'cookiejar':response.meta['keyword'], 'keyword':response.meta['keyword']},
                      callback=self.get_table_9,
                      dont_filter='true')
        # 抽查检查信息


        yield Request(url=self.host+"/ECPS/ccjcgs/gsgs_viewCcjcxx.pt?qyid="+qyid+"&zch="+zch+"&qylx="+qylx+"&showgdxx=true",
                      meta={'qymc':qymc, 'cookiejar':response.meta['keyword'], 'keyword':response.meta['keyword']},
                      callback=self.get_table_10,
                      dont_filter='true')
        # 企业公示


        # 企业年报


        yield Request(url=self.host+"/ECPS/ccjcgs/qygs_ViewQynb.pt?qyid="+qyid+"&zch="+zch+"&qylx="+qylx+"&num="+num + "&showgdxx=true",
                      meta={'qymc':qymc, 'cookiejar':response.meta['keyword'], 'keyword':response.meta['keyword']},
                      callback=self.get_table_11,
                      dont_filter='true')
        # 股东及出资信息


        yield Request(url=self.host+"/ECPS/qygs/gdjcz_viewGdjcz.pt?qyid="+qyid+"&qygsxx=1" + "&showgdxx=true",
                      meta={'qymc':qymc, 'cookiejar':response.meta['keyword'], 'keyword':response.meta['keyword']},
                      callback=self.get_table_12,
                      dont_filter='true')
        # 股权变更信息


        yield Request(url=self.host+"/ECPS/qygs/gqbg_viewGqbg.pt?qyid="+qyid+"&qygsxx=1" + "&showgdxx=true",
                      meta={'qymc':qymc, 'cookiejar':response.meta['keyword'], 'keyword':response.meta['keyword']},
                      callback=self.get_table_13,
                      dont_filter='true')
        # 行政许可信息


        yield Request(url=self.host+"/ECPS/qygs/xzxk_viewXzxk.pt?qyid="+qyid+"&qygsxx=1"+"&zch="+zch+"&qylx="+qylx+"&num="+num+"&qymc="+qymc + "&showgdxx=true",
                      meta={'qymc':qymc, 'cookiejar':response.meta['keyword'], 'keyword':response.meta['keyword']},
                      callback=self.get_table_14,
                      dont_filter='true')
        # 知识产权出质登记信息


        yield Request(url=self.host+"/ECPS/qygs/zscqczdj_viewZscqczdj.pt?qyid="+qyid+"&qygsxx=1"+"&zch="+zch+"&qylx="+qylx+"&num="+num+"&qymc="+qymc + "&showgdxx=true",
                      meta={'qymc':qymc, 'cookiejar':response.meta['keyword'], 'keyword':response.meta['keyword']},
                      callback=self.get_table_15,
                      dont_filter='true')
        # 行政处罚信息


        yield Request(url=self.host+"/ECPS/qygs/xzcf_viewXzcf.pt?qyid="+qyid+"&qygsxx=1" + "&showgdxx=true",
                      meta={'qymc':qymc, 'cookiejar':response.meta['keyword'], 'keyword':response.meta['keyword']},
                      callback=self.get_table_16,
                      dont_filter='true')
        # 其他公示


        # 行政许可


        yield Request(url=self.host+"/ECPS/ccjcgs/qygs_ViewQtbmxzxk.pt?qyid="+qyid+"&zch="+zch+"&qylx="+qylx+"&num="+num,
                      meta={'qymc':qymc, 'cookiejar':response.meta['keyword'], 'keyword':response.meta['keyword']},
                      callback=self.get_table_17,
                      dont_filter='true')
        # 行政处罚信息


        yield Request(url=self.host+"/ECPS/ccjcgs/qygs_ViewQtbmxzcf.pt?qyid="+qyid+"&zch="+zch+"&qylx="+qylx+"&num="+num,
                      meta={'qymc':qymc, 'cookiejar':response.meta['keyword'], 'keyword':response.meta['keyword']},
                      callback=self.get_table_18,
                      dont_filter='true')
        # 司法公示


        # 司法股权冻结信息


        yield Request(url=self.host+"/ECPS/sfxz/gqdj_gqdjList.pt?qyid="+qyid+"&zch="+zch+"&qylx="+qylx+"&num="+num,
                      meta={'qymc':qymc, 'cookiejar':response.meta['keyword'], 'keyword':response.meta['keyword']},
                      callback=self.get_table_19,
                      dont_filter='true')
        # 司法股东变更登记信息


        yield Request(url=self.host+"/ECPS/sfxz/gdbg_gdbgList.pt?qyid="+qyid+"&zch="+zch+"&qylx="+qylx+"&num="+num,
                      meta={'qymc':qymc, 'cookiejar':response.meta['keyword'], 'keyword':response.meta['keyword']},
                      callback=self.get_table_20,
                      dont_filter='true')

    def get_table_1(self, response):
        mo = MongoItem()
        mo['_id'] = response.meta['qymc']
        startends = [['基本信息</th>', '</table>', '工商公示-基本信息']]
        mo['document'] = html_to_list_dict(response.body, startends)
        yield mo

    def get_table_2(self, response):
        mo = MongoItem()
        mo['_id'] = response.meta['qymc']
        startends = [['发起人信息', '</table>', '工商公示-股东信息']]
        mo['document'] = html_to_list_dict(response.body, startends)
        yield mo

    def get_table_3(self, response):
        mo = MongoItem()
        mo['_id'] = response.meta['qymc']
        startends = [['变更信息</th>', '</table>', '工商公示-变更信息']]
        mo['document'] = html_to_list_dict(response.body, startends)
        yield mo

    def get_table_4(self, response):
        mo = MongoItem()
        mo['_id'] = response.meta['qymc']
        startends = [['主要人员信息', '</table>', '工商公示-主要人员信息'],
                     ['分支机构信息</th>', '</table>', '工商公示-分支机构信息'],
                     ['清算信息</th>', '</table>', '工商公示-清算信息']]
        mo['document'] = html_to_list_dict(response.body, startends)
        yield mo

    def get_table_5(self, response):
        mo = MongoItem()
        mo['_id'] = response.meta['qymc']
        startends = [['股权出质登记信息</th>', '</table>', '工商公示-股权出质登记信息']]
        mo['document'] = html_to_list_dict(response.body, startends)
        yield mo

    def get_table_6(self, response):
        mo = MongoItem()
        mo['_id'] = response.meta['qymc']
        startends = [['动产抵押登记信息</th>', '</table>', '工商公示-动产抵押登记信息']]
        mo['document'] = html_to_list_dict(response.body, startends)
        yield mo

    def get_table_7(self, response):
        mo = MongoItem()
        mo['_id'] = response.meta['qymc']
        startends = [['经营异常信息</th>', '</table>', '工商公示-经营异常信息']]
        mo['document'] = html_to_list_dict(response.body, startends)
        yield mo

    def get_table_8(self, response):
        mo = MongoItem()
        mo['_id'] = response.meta['qymc']
        startends = [['严重违法信息</th>', '</table>', '工商公示-严重违法信息']]
        mo['document'] = html_to_list_dict(response.body, startends)
        yield mo

    def get_table_9(self, response):
        mo = MongoItem()
        mo['_id'] = response.meta['qymc']
        startends = [['行政处罚信息</th>', '</table>', '工商公示-行政处罚信息']]
        mo['document'] = html_to_list_dict(response.body, startends)
        yield mo

    def get_table_10(self, response):
        mo = MongoItem()
        mo['_id'] = response.meta['qymc']
        startends = [['抽查检查信息</th>', '</table>', '工商公示-抽查检查信息']]
        mo['document'] = html_to_list_dict(response.body, startends)
        yield mo

    def get_table_11(self, response):
        mo = MongoItem()
        mo['_id'] = response.meta['qymc']
        startends = [['企业年报</th>', '</table>', '企业公示-企业年报']]
        mo['document'] = html_to_list_dict(response.body, startends)
        yield mo

        # 年报链接


        annual_link = response.xpath("//a/@href").extract()
        for url in annual_link:
            if u'qygs_ViewQynbDetail' in url:
                yield Request(url = self.host + url,
                              meta={'company_name':response.meta['qymc'], 'cookiejar':response.meta['keyword'], 'keyword':response.meta['keyword']},
                              callback= self.annual_info,
                              dont_filter='true')

    def get_table_12(self, response):
        mo = MongoItem()
        mo['_id'] = response.meta['qymc']
        startends = [['股东及出资信息</th>', '</table>', '企业公示-股东及出资信息'],
                     ['变更信息</th>', '</table>', '企业公示-变更信息']]
        mo['document'] = html_to_list_dict(response.body, startends)
        yield mo

    def get_table_13(self, response):
        mo = MongoItem()
        mo['_id'] = response.meta['qymc']
        startends = [['股权变更信息</th>', '</table>', '企业公示-股权变更信息']]
        mo['document'] = html_to_list_dict(response.body, startends)
        yield mo

    def get_table_14(self, response):
        mo = MongoItem()
        mo['_id'] = response.meta['qymc']
        startends = [['行政许可信息</th>', '</table>', '企业公示-行政许可信息']]
        mo['document'] = html_to_list_dict(response.body, startends)
        yield mo

    def get_table_15(self, response):
        mo = MongoItem()
        mo['_id'] = response.meta['qymc']
        startends = [['知识产权出质登记信息</th>', '</table>', '企业公示-知识产权出质登记信息']]
        mo['document'] = html_to_list_dict(response.body, startends)
        yield mo

    def get_table_16(self, response):
        mo = MongoItem()
        mo['_id'] = response.meta['qymc']
        startends = [['行政处罚信息</th>', '</table>', '企业公示-行政处罚信息']]
        mo['document'] = html_to_list_dict(response.body, startends)
        yield mo

    def get_table_17(self, response):
        mo = MongoItem()
        mo['_id'] = response.meta['qymc']
        startends = [['行政许可信息</th>', '</table>', '其他公示-行政许可信息']]
        mo['document'] = html_to_list_dict(response.body, startends)
        yield mo

    def get_table_18(self, response):
        mo = MongoItem()
        mo['_id'] = response.meta['qymc']
        startends = [['行政处罚信息</th>', '</table>', '其他公示-行政处罚信息']]
        mo['document'] = html_to_list_dict(response.body, startends)
        yield mo

    def get_table_19(self, response):
        mo = MongoItem()
        mo['_id'] = response.meta['qymc']
        startends = [['司法股权冻结信息</th>', '</table>', '司法公示-司法股权冻结信息']]
        mo['document'] = html_to_list_dict(response.body, startends)
        yield mo

    def get_table_20(self, response):
        mo = MongoItem()
        mo['_id'] = response.meta['qymc']
        startends = [['司法股东变更登记信息</th>', '</table>', '司法公示-司法股东变更登记信息']]
        mo['document'] = html_to_list_dict(response.body, startends)
        yield mo

    # 年报信息


    def annual_info(self,response):
        name = response.xpath("//div[1]/table[1]/tr[1]/th/text()").extract_first()
        if u"红色为修改过的信息项" in name:
            name = re.findall(u'([\s\S]+)红色为修改过的信息项',name)[0].strip()

        mo = MongoItem()
        mo['_id'] = response.meta['company_name']
        mo['document'] = tables_to_dict(response.body, name, 'detailsList')
        yield mo

    # 排除上次已经跑过的词语，创建词库列表


    def creatlist(self,startnum,endnum):
        # 清空"跳过词语"的记录


        fp = open(self.skip_record_path, 'w')
        fp.close()
        a = 1

        f = open(self.ppl_path, 'rb')
        list = []
        while a < endnum:
            line = f.readline()
            if not line:
                    break
            if startnum <= a :
                line = line.replace('\'','')
                line = line.replace('\n','')
                if line not in self.already:
                    list.append(line)
                else:
                    op = open(self.skip_record_path, 'a')
                    op.write(line+'\n')
                    op.close()
            a += 1
        return list

    # 读取上次已经跑过的词语


    def wordcount(self):
        already = []
        for line in open(self.realtime_record_path, 'rb'):
            if line.strip():
                vars = line.split(':', 2)
                if vars:
                    word = vars[0]
                    if word not in already:
                        already.append(word)
        return already